package cn.doitedu.dwetl.etl

import cn.doitedu.commons.utils.SparkUtil
import org.apache.spark.rdd.RDD

/**
 * @Title: ${file_name}
 * @Package ${package_name}
 * @Description: 数仓增量导入的数据，滚动合并到全量表中
 * @Author hunter@doitedu.cn
 * @date 2020/8/616:16     
 */
object AdditionCombine {
  def main(args: Array[String]): Unit = {

    val spark = SparkUtil.getSparkSession("stu表增量合并")

    // 加载上一个全量表
    val oldWhole = spark.read.textFile("/sqoopdata/add/2020-08-04/")

    // 加载当日增量数据
    val addition = spark.read.textFile("/sqoopdata/add/2020-08-05/")

    val res: RDD[String] = oldWhole.union(addition).rdd.groupBy(line=>line.split(",")(0))
      .map(tp=>{
        tp._2.toList.maxBy(line=>line.split(",")(5))
      })


    res.foreach(println)


    spark.close()
  }

}
